package com.chance.cc.crawler.development.scripts.mogu;

import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.HttpRequestBody;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.filter.FilterInfo;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.development.scripts.allfeild.AICCommonField;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.util.*;

/**
 * @ClassName mogujie
 * @Description TODO
 * @Author songding
 * @Date 2021/10/15 9:40
 * @Version 1.0
 **/
public class MoguCrawlerScript extends CrawlerCommonScript {
    private static Logger log = LoggerFactory.getLogger(MoguCrawlerScript.class);
    public static final String domain = "mogu";
    private static final String site = "article";
    private static final String RECORD_AGAIN_REQUEST = "record_again_request";

    private static final String referer = "https://list.mogu.com/";
    private static final String moguUrl = "https://api.mogu.com/h5/mwp.pagani.search/19/?";
    private static final String mw_ckey = "pc-search-wall";
    private static final String mw_appkey = "100028";
    private static final String mw_ttidCookie = "NMMain%40mgj_pc_1.0";//得到cookie使用的ttid
    private static final String mw_ttidlist = "NMMain%40mgj_h5_1.0";//得到列表页使用的ttid
    private static final String mw_ttidlist1 ="NMMain@mgj_h5_1.0";
    private static  String mw_t =String.valueOf(System.currentTimeMillis());
    private static  String mw_uuid= UUID.randomUUID().toString();
    private static final String mw_h5_os= "Android";
    private static final String callbackCookie= "mwpCb1";
    private static final String callbackList = "mwpCb2";
    private static  String _= String.valueOf(System.currentTimeMillis());
    private static final String commentUrl = "https://api.mogu.com/h5/mwp.RateWeb.commonRateList/v1/?";

    private static final String moguListUrl = "https://list.mogu.com/search/goods?q=";
    private static final String moguListRegulars = "https://list.mogu.com/search/goods\\?q=\\S*";
    private static final String moguUrlRegulars = "https://api.mogu.com/h5/mwp.pagani.search/19/\\S*";
    private static final String commentRegulars = "https://api.mogu.com/h5/mwp.RateWeb.commonRateList/v1/\\S*";

    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord requestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        List<CrawlerRecord> parseLinks = new ArrayList<>();
        if (supportSourceRecords != null){
            for (CrawlerRequestRecord supportSourceRecord :supportSourceRecords){
                String url = supportSourceRecord.getHttpRequest().getUrl();
                if (url.contains("keys")){
                    HttpPage internalDownloadPage = supportSourceRecord.getInternalDownloadPage();
                    Json json = internalDownloadPage.getJson();
                    String msg = json.jsonPath($_type+".msg").get();
                    if (!"success".equals(msg)){
                        log.error("keyword page [{}] download error!", internalDownloadPage.getRequest().getUrl());
                        return parseLinks;
                    }
                    List<String> all = json.jsonPath($_type + ".content").all();
                    for (String data : all) {
                        JSONObject jsonObject = JSONObject.parseObject(data);
                        String keyword = jsonObject.getString("keyword");
                        String keywords = null;
                        try {
                           keywords =  URLEncoder.encode(keyword,"utf-8");
                        } catch (UnsupportedEncodingException e) {
                            e.printStackTrace();
                        }
                        String startUrl = moguListUrl + keywords;
                        CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                                .itemPageRequest(requestRecord)
                                .httpUrl(startUrl)
                                .releaseTime(System.currentTimeMillis())
                                .needParsed(true)
                                .copyResultTags()
                                .copyBizTags()
                                .build();
                        record.tagsCreator().bizTags().addCustomKV("q",keywords);
                        record.tagsCreator().bizTags().addCustomKV("keywords",keyword);
                        parseLinks.add(record);
                    }
                }
            }
        }

       /* if (!url.matches(moguListRegulars)){
            return parseLinks;
        }
        String q = requestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("q");
        String data = "{\"page\":1,\"pageSize\":24,\"sort\":\"pop\",\"ratio\":\"3:4\",\"cKey\":\"pc-search-wall\",\"q\":\""+q+"\",\"ptp\":\"31.DGODi.0.0.2sT0NnmK\"}";
        data = URLEncoder.encode(data);
        String uuid = mw_uuid;
        String cookieUrl = moguUrl+"data="+data+"&"+"mw-ckey="+mw_ckey+"&"+"mw-appkey="+mw_appkey+"&"+"mw-ttid="+mw_ttidCookie+"&"+"mw-t="+mw_t+"&"+"mw-uuid="+uuid+"&"+"mw-h5-os="+mw_h5_os+"&"+"callback="+callbackCookie+"&"+"_="+_;
        CrawlerRequestRecord cookieRecord = CrawlerRequestRecord.builder()
                .itemPageRequest(requestRecord)
                .httpUrl(cookieUrl)
                .releaseTime(System.currentTimeMillis())
                .httpCookie("__mgjuuid",uuid)
                .httpHead("referer",referer)
                .copyBizTags()
                .copyResultTags()
                .build();
        cookieRecord.getHttpConfig().setDisableCookie(true);
        parseLinks.add(cookieRecord);*/
        return parseLinks;
    }

        @Override
    public void initUrlRegulars() {
        addUrlRegular(moguListRegulars);
        addUrlRegular(moguUrlRegulars);
        addUrlRegular(commentRegulars);
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerRequestRecord> parseLinks = new ArrayList<>();
        if (page.getStatusCode() != 200 ||  !page.isDownloadSuccess()){
            log.error("page == [{}] || statusCode != 200 and error page = "+ page.getStatusCode());
            crawlerRecord.setNeedWashPage(false);
            if (page.getStatusCode() != 404){
                recordAgainRequest(crawlerRecord,parseLinks);
                crawlerRecord.setNeedWashPage(false);
                return  parseLinks;
            }
        }
        String url = crawlerRecord.getHttpRequest().getUrl();
        if (url.matches(moguUrlRegulars) ||url.matches(moguListRegulars)){
            this.getListUrl(crawlerRecord,page,parseLinks);
            this.getArticleUrl(crawlerRecord,page,parseLinks);
        }
        if (url.matches(commentRegulars)){
            this.getTurnComment(crawlerRecord,page,parseLinks);
        }
        return parseLinks;
    }

    private void getTurnComment(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        String rawText = page.getRawText();
        rawText = rawText.substring(7,rawText.length()-1);
        page.setRawText(rawText);
        String api = page.getJson().jsonPath($_type+".api").get();
        String v = page.getJson().jsonPath($_type+".v").get();
        String mbook = null;
        try{
            mbook = page.getJson().jsonPath($_type+".data.mbook").get();
        }catch (Exception e){
            log.error("no page");
            return;
        }

        String itemId = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("itemId");
        String _mwp_h5_token = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("_mwp_h5_token");
        String mwp_h5_token_enc = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("mwp_h5_token_enc");
        String data = "{\"mbook\":"+mbook+"\"\",\"sort\":0,\"source\":\"item\",\"first\":\"true\",\"itemId\":\""+itemId+"\",\"labelIds\":\"-1\",\"topRateTagId\":1}";
        String mw_t = "1634528499972";
        String uuid = "3877d405-0f59-460f-ac1f-526b5f108ae2";
        String mw_sign = this.getCommentMW_sign(data,api,mw_t,uuid,_mwp_h5_token,v);
        data = URLEncoder.encode(data);
        String commentUrl = this.commentUrl+"data="+data+"&mw-appkey="+mw_appkey+"&mw-ttid="+mw_ttidlist1+"&mw-t="+mw_t+"&mw-uuid="+uuid+"&mw-h5-os="+mw_h5_os+"&mw-sign="+mw_sign+"&callback="+callbackList+"&_="+_;
        CrawlerRequestRecord commentRecord = CrawlerRequestRecord.builder()
                .itemPageRequest(crawlerRecord)
                .httpUrl(commentUrl)
                .releaseTime(System.currentTimeMillis())
                .httpHead("referer",referer)
                .httpHead("cookie","__mgjuuid="+uuid+"; _mwp_h5_token="+_mwp_h5_token+"; _mwp_h5_token_enc="+mwp_h5_token_enc)
                .copyBizTags()
                .copyResultTags()
                .build();
        commentRecord.getHttpConfig().setDisableCookie(true);
        commentRecord.tagsCreator().bizTags().addCustomKV("itemId",itemId);
        parseLinks.add(commentRecord);
    }

    private void getArticleUrl(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        if (crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("_mwp_h5_token") == null){
            return;
        }
        String rawText = page.getRawText();
        rawText = rawText.substring(7,rawText.length()-1);
        page.setRawText(rawText);
        List<String> all = page.getJson().jsonPath($_type + ".data.wall.docs").all();
        String mwp_h5_token = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("_mwp_h5_token");
        String mwp_h5_token_enc = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("_mwp_h5_token_enc");
        crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("");
        for (String str : all){
            JSONObject jsonObject = JSONObject.parseObject(str);
            String tradeItemId = jsonObject.getString("tradeItemId");
            String acm = jsonObject.getString("acm");
            String url = "https://shop.mogu.com/detail/"+tradeItemId+"?"+acm;
            String data = "{\"mbook\":\"\",\"sort\":0,\"source\":\"item\",\"first\":\"true\",\"itemId\":\""+tradeItemId+"\",\"labelIds\":\"-1\",\"topRateTagId\":1}";
            String mw_t = "1634528499972";
            String uuid = "3877d405-0f59-460f-ac1f-526b5f108ae2";
            String api= "mwp.RateWeb.commonRateList";
            String v = "v1";
            String mw_sign = this.getCommentMW_sign(data,api,mw_t,uuid,mwp_h5_token,v);
            data = URLEncoder.encode(data);
            String commentUrl = this.commentUrl+"data="+data+"&mw-appkey="+mw_appkey+"&mw-ttid="+mw_ttidlist1+"&mw-t="+mw_t+"&mw-uuid="+uuid+"&mw-h5-os="+mw_h5_os+"&mw-sign="+mw_sign+"&callback="+callbackList+"&_="+_;
            CrawlerRequestRecord commentRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(commentUrl)
                    .releaseTime(System.currentTimeMillis())
                    .httpHead("referer",referer)
                    .httpHead("cookie","__mgjuuid="+uuid+"; _mwp_h5_token="+mwp_h5_token+"; _mwp_h5_token_enc="+mwp_h5_token_enc)
                    .copyBizTags()
                    .copyResultTags()
                    .build();
            commentRecord.getHttpConfig().setDisableCookie(true);
            commentRecord.tagsCreator().bizTags().addCustomKV("itemId",tradeItemId);
            commentRecord.tagsCreator().bizTags().addCustomKV("url",url);
            parseLinks.add(commentRecord);
        }

    }

    private String getCommentMW_sign(String data,String api,String mw_t,String uuid,String token,String v) {
        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());
        HttpConfig httpConfig = HttpConfig.me(domain);
        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setResponseCharset("UTF-8");
        httpRequest.setMethod("post");
        httpRequest.setUrl("http://192.168.1.210:8899/encrypt/moguArticle");
        Map<String,Object> bodyMap = new HashMap<>();
        bodyMap.put("data",data);
        bodyMap.put("mw_appkey",mw_appkey);
        bodyMap.put("httpApi",api);
        bodyMap.put("mw_ttid",mw_ttidlist1);
        bodyMap.put("mw_t",mw_t);
        bodyMap.put("mw_uuid",uuid);
        bodyMap.put("mw_h5_os",mw_h5_os);
        bodyMap.put("cookieToken",token);
        bodyMap.put("v",v);
        HttpRequestBody form = HttpRequestBody.form(bodyMap, "UTF-8");
        httpRequest.setRequestBody(form);
        HttpPage download = downloader.download(httpRequest, httpConfig);
        String rawText = download.getRawText();
        return rawText;
    }

    private void getListUrl(CrawlerRequestRecord crawlerRecord, HttpPage page,List<CrawlerRequestRecord> parseLinks) {
        Map<String, String> cookies = crawlerRecord.getHttpRequest().getCookies();
        String api = "mwp.pagani.search";
        String v = "19";
        String token = "ef530c2910cfec7029493406a754467e_1637204098770";//null;
        String encToken = "508f79a84a1b667575e2c3b4eb56302f";
        if (crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("_mwp_h5_token") == null){
            String rawText = page.getRawText();
            rawText = rawText.substring(7, rawText.length() - 1);
            page.setRawText(rawText);
           // api = page.getJson().jsonPath($_type + ".api").get();
            //v = page.getJson().jsonPath($_type + ".v").get();
            //token = page.getJson().jsonPath($_type + ".token").get();
           // encToken = page.getJson().jsonPath($_type + ".encToken").get();
        }
        //准备得到列表页
        KVTag key = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("pageSize");
        int pageSize = 1;
        if (key == null){
            pageSize = 1;
        }else{
            pageSize =  (Integer) key.getVal();
        }
         if (crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("_mwp_h5_token") != null){
             token = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("_mwp_h5_token");
             encToken = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("_mwp_h5_token_enc");
         }
        String q = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("q");
        //String data = "{\"page\":"+pageSize+",\"pageSize\":24,\"sort\":\"pop\",\"ratio\":\"3:4\",\"cKey\":\"pc-search-wall\",\"q\":\""+q+"\",\"ptp\":\"31.DGODi.0.0.2sT0NnmK\"}";
        String data ="{\"page\":"+pageSize+",\"pageSize\":24,\"sort\":\"pop\",\"ratio\":\"3:4\",\"cKey\":\"pc-search-wall\",\"q\":\""+q+"\"}";
        String time = String.valueOf(System.currentTimeMillis());
        String mw_t  = time;
        //String mw_t ="1637288656735"; //this.mw_t;
        String uuid = "3877d405-0f59-460f-ac1f-526b5f108ae2";//this.mw_uuid;
        String mw_sign = this.getMw_sign(data,api,mw_t,uuid,token,v);
        data = URLEncoder.encode(data);
        String listUrl = moguUrl+"data="+data+"&mw-ckey="+mw_ckey+"&mw-appkey="+mw_appkey+"&mw-ttid="+mw_ttidlist+"&mw-t="+mw_t+"&mw-uuid="+uuid+"&mw-h5-os="+mw_h5_os+"&mw-sign="+mw_sign+"&callback="+callbackList+"&_=1634362729517";
        CrawlerRequestRecord listRecord = CrawlerRequestRecord.builder()
                .itemPageRequest(crawlerRecord)
                .httpUrl(listUrl)
                .releaseTime(System.currentTimeMillis())
                .httpHead("referer",referer)
                .httpHead("cookie","__mgjuuid="+uuid+"; _mwp_h5_token="+token+"; _mwp_h5_token_enc="+encToken)
                .copyBizTags()
                .copyResultTags()
                .build();
        listRecord.getHttpConfig().setDisableCookie(true);
        listRecord.tagsCreator().bizTags().addCustomKV("_mwp_h5_token",token);
        listRecord.tagsCreator().bizTags().addCustomKV("_mwp_h5_token_enc",encToken);
        listRecord.tagsCreator().bizTags().addCustomKV("pageSize",pageSize+1);
        parseLinks.add(listRecord);
    }

    private String getMw_sign(String data,String api,String mw_t,String uuid,String token,String v) {
        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());
        HttpConfig httpConfig = HttpConfig.me(domain);
        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setResponseCharset("UTF-8");
        httpRequest.setMethod("post");
        httpRequest.setUrl("http://192.168.1.210:8899/encrypt/mogulist");
        Map<String,Object> bodyMap = new HashMap<>();
        bodyMap.put("data",data);
        bodyMap.put("mw_appkey",mw_appkey);
        bodyMap.put("httpApi",api);
        bodyMap.put("mw_ttid",mw_ttidlist1);
        bodyMap.put("mw_t",mw_t);
        bodyMap.put("mw_uuid",uuid);
        bodyMap.put("mw_h5_os",mw_h5_os);
        bodyMap.put("cookieToken",token);
        bodyMap.put("v",v);
        HttpRequestBody form = HttpRequestBody.form(bodyMap, "UTF-8");
        httpRequest.setRequestBody(form);
        HttpPage download = downloader.download(httpRequest, httpConfig);
        String rawText = download.getRawText();
        return rawText;
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> dataList = new ArrayList<>();
        String url = crawlerRecord.getHttpRequest().getUrl();
        if (crawlerRecord.tagsCreator().resultTags().hasDataType(CrawlerEnum.CrawlerDataType.article)){
            if (url.matches(moguUrlRegulars)){
                this.washArticle(crawlerRecord,page,dataList);
            }
        }
        if (crawlerRecord.tagsCreator().resultTags().hasDataType(CrawlerEnum.CrawlerDataType.comment)){
            if (url.matches(commentRegulars)){
                this.washComment(crawlerRecord,page,dataList);
            }
        }
        return dataList;
    }

    private void washComment(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerData> dataList) {
        List<String> all = page.getJson().jsonPath($_type + ".data.list").all();
        String url = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("url");
        for (String str : all){
            JSONObject jsonObject = JSONObject.parseObject(str);
            String itemId = jsonObject.getString("itemId");
            String rateId = jsonObject.getString("rateId");
            String images = jsonObject.getString("images");
            String content = jsonObject.getString("content");
            String created = jsonObject.getString("created")+"000";
            JSONObject user = jsonObject.getJSONObject("user");
            String uid = user.getString("uid");
            String uname = user.getString("uname");
            if (!isDateRange(crawlerRecord,Long.valueOf(created))){
                return;
            }
            CrawlerData commentData = CrawlerData.builder()
                    .data(crawlerRecord,page)
                    .dataId(StringUtils.joinWith("-",crawlerRecord.getDomain(),CrawlerEnum.CrawlerDataType.comment,rateId))
                    .parentId(StringUtils.joinWith("-",crawlerRecord.getDomain(),CrawlerEnum.CrawlerDataType.article,itemId))
                    .url(url)
                    .releaseTime(Long.valueOf(created))
                    .addContentKV(AICCommonField.Field_Author_Id,uid)
                    .addContentKV(AICCommonField.Field_Author,uname)
                    .addContentKV(AICCommonField.Field_Content,content)
                    .addContentKV(AICCommonField.Field_Images,images)
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.comment)
                    .build();
            dataList.add(commentData);
        }
    }

    private void washArticle(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerData> dataList) {
        if (crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("_mwp_h5_token") == null){
            return;
        }
        List<String> all = page.getJson().jsonPath($_type + ".data.wall.docs").all();
        String monthly = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("monthly");
        for (String str : all){
            JSONObject jsonObject = JSONObject.parseObject(str);
            String tradeItemId = jsonObject.getString("tradeItemId");
            String acm = jsonObject.getString("acm");
            String url = "https://shop.mogu.com/detail/"+tradeItemId+"?"+acm;
            String title = jsonObject.getString("title");
            String img = jsonObject.getString("img");
            String price = jsonObject.getString("price");
            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRecord,page)
                    .dataId(StringUtils.joinWith("-",crawlerRecord.getDomain(),CrawlerEnum.CrawlerDataType.article,tradeItemId))
                    .url(url)
                    .releaseTime(System.currentTimeMillis())
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                    .addContentKV(AICCommonField.Field_Urls,url)
                    .addContentKV(AICCommonField.Field_Title,title)
                    .addContentKV(AICCommonField.Field_Images,img)
                    .addContentKV(AICCommonField.Field_Produce_Price,price)
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                    .build();
            dataList.add(crawlerData);
            if (monthly!=null){
                CrawlerData crawlerUrl = CrawlerData.builder()
                        .data(crawlerRecord,page)
                        .dataId(StringUtils.joinWith("-",crawlerRecord.getDomain(),CrawlerEnum.CrawlerDataType.article,tradeItemId))
                        .url(url)
                        .releaseTime(System.currentTimeMillis())
                        .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                        .addContentKV(AICCommonField.Field_Urls,url)
                        .build();
                dataList.add(crawlerUrl);
            }
        }
    }


    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        return crawlerRecord.tagsCreator().bizTags().site().equals(site);
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }

    @Override
    public String domain() {
        return domain;
    }
    private static void recordAgainRequest(CrawlerRequestRecord crawlerRequestRecord,List<CrawlerRequestRecord> parseList){
        int count = 0;
        String url = crawlerRequestRecord.getHttpRequest().getUrl();
        if (crawlerRequestRecord.tagsCreator().bizTags().hasKVTag(RECORD_AGAIN_REQUEST)){
            count = Integer.valueOf(crawlerRequestRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(RECORD_AGAIN_REQUEST));
            if (count >= 10){
                log.error("url excessive number of repeated downloads this url = "+url);
            }
        }else{
            count = 1;
        }
        CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                .itemPageRequest(crawlerRequestRecord)
                .releaseTime(System.currentTimeMillis())
                .httpUrl(url)
                .recordKey(crawlerRequestRecord.getRecordKey()+count)
                .copyResultTags()
                .copyBizTags()
                .build();

        record.getHttpRequest().setCookies(crawlerRequestRecord.getHttpRequest().getCookies());
        record.getHttpRequest().setExtras(crawlerRequestRecord.getHttpRequest().getExtras());
        record.getHttpRequest().setHeaders(crawlerRequestRecord.getHttpRequest().getHeaders());
        record.setNeedParsedPage(crawlerRequestRecord.isNeedParsedPage());
        record.setNeedWashPage(crawlerRequestRecord.isNeedWashPage());
        record.tagsCreator().bizTags().addCustomKV(RECORD_AGAIN_REQUEST,count++);
        parseList.add(record);

    }
    /**
     * 判断是否在时间范围内
     * @param crawlerRequestRecord
     * @return
     */
    private boolean isDateRange(CrawlerRequestRecord crawlerRequestRecord,Long releaseTimeToLong){
        boolean isRange = false;
        CrawlerEnum.CrawlerRecordFilter filter = crawlerRequestRecord.getFilter();
        if (filter == CrawlerEnum.CrawlerRecordFilter.keyOrDateRange || filter == CrawlerEnum.CrawlerRecordFilter.dateRange) {
            List<FilterInfo> filterInfos = crawlerRequestRecord.getFilterInfos();
            Long startTime = null;
            Long endTime = null;
            for (FilterInfo filterInfo : filterInfos) {
                if (filterInfo.getFilter() == CrawlerEnum.CrawlerRecordFilter.dateRange) {
                    long[] dateAllowRange = filterInfo.getDateAllowRange();
                    int hourFromNow = filterInfo.getHourFromNow();
                    if (dateAllowRange != null) {
                        startTime = dateAllowRange[0];
                        endTime = dateAllowRange[1];
                    }else if(hourFromNow != 0){
                        endTime = System.currentTimeMillis()-60000;//系统时间减去一分钟
                        startTime = endTime - DateUtils.MILLIS_PER_HOUR * hourFromNow;
                    }
                }
            }
            if(startTime != null && releaseTimeToLong != 0 && startTime <= releaseTimeToLong && releaseTimeToLong <= endTime){
                isRange = true;
            }
        }else{
            isRange = true;
        }
        return isRange;
    }
}
